--- title: Atmospheric Correction keywords: fastai sidebar: home_sidebar summary: "Out in the field, the OpenHSI camera measures light as it is reflected off surfaces. This light from the sun has absorption features from molecules/aerosols in the atmosphere and scattering. After converting the raw digital number counts into radiance" description: "Out in the field, the OpenHSI camera measures light as it is reflected off surfaces. This light from the sun has absorption features from molecules/aerosols in the atmosphere and scattering. After converting the raw digital number counts into radiance" nb_path: "nbs/03_atmos.ipynb" ---
Find the closest station at http://weather.uwyo.edu/upperair/sounding.html
to use the atmospheric sounding on the day.
find the station number and region code (for example, south pacific is pac and new zealand is nz)
Default is Willis Island in Queensland, Australia.
https://py6s.readthedocs.io/en/latest/helpers.html#importing-atmospheric-profiles-from-radiosonde-data
{% include warning.html content='Py6S states that custom aerosol profiles can be input but this is broken and there is no easy fix. Don’t waste a day trying... ' %}
Py6S provides a method to loop over an array and return the 6SV result. However, it does not have a progress bar. For something that can take several minutes, not knowing how long it will take is genuinely frustrating. Therefore, we provide a modified version of SixSHelpers.Wavelengths.run_wavelengths called Model6SV.run_wavelengths that has a progress bar.
To use 6SV, Py6S will write input files and parse the output files. All this I/O is slow but we got to deal with it. Py6S is clever about it though and uses threading to do some compute while waiting for files. The modified version follows the same method except with the addition of a callback to update the progress bar when interations are done.
model = Model6SV(wavelength_array=np.arange(350, 1000, 10),sixs_path="../openhsi_pre/assets/6SV1.1/sixsV1.1")
model.show()
dc = DataCube(processing_lvl=-1)
dc.load_nc("../load_multiple/2021-05-26 03_26_26.011211 radiance float32.nc",old_style=True)
dc.show("bokeh",robust=True).opts(fontsize={'title': 15, 'labels': 14, 'xticks': 10, 'yticks': 10,})
model = Model6SV(wavelength_array=dc.binned_wavelengths,sixs_path="../openhsi_pre/assets/6SV1.1/sixsV1.1")
model.show()
spec_matcher = SpectralMatcher(speclib_path="assets/speclib.pkl",model_6SV=model)
%time spec_matcher.topk_spectra(dc.dc.data[450,1500,:],5,refine=True)
spec_matcher.show()
{% include warning.html content='This web page was generated from a Jupyter notebook and not all interactivity will work on this website. ' %}
elc = ELC(nc_path="../load_multiple/2021-05-26 03_26_26.011211 radiance float32.nc",old_style=True,speclib_path="assets/speclib.pkl",model_6SV=model)
elc()
df = pd.read_pickle("assets/openhsi_radiance_select_targets.pkl")
df.plot(x='wavelength (nm)', y=df.columns[1:],figsize=(8,6),
xlabel="wavelength (nm)",ylabel="radiance (uW/cm^2/sr/nm)",
ylim=(0,None)).legend(loc="upper right",fontsize='xx-small')
speclib2 = elc.speclib.copy()
speclib2.rename(columns={"gray_small":"gray small","green_small":"green","cyan_small":"cyan","red_small":"magenta"},inplace=True)
name_dict = {"gray uni":"gray small","cyan uni":"cyan","green uni":"green",
"magenta uni":"magenta","charcoal":"charcoal","gray":"gray","blue":"blue",
"blue takeoff":"blue_take_off"}
name_dict2 = dict((v,k) for k,v in name_dict.items())
from pathlib import Path
@patch
def show(self:SpectralMatcher,is_ref:bool=False,ref_est:bool=False):
if self.topk_idx is not None:
hv_curves = []
if not is_ref:
hv_curves.append( hv.Curve(zip(self.wavelengths,self.last_spectra),label="tap point") )
elif ref_est:
hv_curves.append( hv.Curve(zip(self.wavelengths,self.last_spectra/(self.model_6SV.radiance/10)),label="6SV estimate") )
for l in self.sim_df["label"]:
alpha = self.sim_df[self.sim_df["label"]==l]["score"].to_numpy()
alpha = 0.2 if len(alpha) == 0 else alpha[0]
thresh = 0.94 if self.refine else 0.99
alpha = 0.2 if alpha < thresh else 0.9
temp = self.speclib_ref.copy() if is_ref else self.spectra.copy()
temp.insert(0,"wavelength",self.wavelengths)
hv_curves.append( hv.Curve(temp,kdims="wavelength",vdims=l,label=l).opts(alpha=alpha) )
return hv.Overlay(hv_curves).opts(width=1000, height=600,xlabel="wavelength (nm)",ylabel="reflectance")
else:
temp = self.speclib_ref.copy() if is_ref else self.spectra.copy()
temp.insert(0,"wavelength",self.wavelengths)
curve_list = [hv.Curve(temp,kdims="wavelength",vdims=i,label=i) for i in temp.columns[1:] if (np.sum(temp[i]>1.)>0)]
print(curve_list)
if getattr(self,"last_spectra",None):
return (hv.Overlay(curve_list)*hv.Curve(zip(self.wavelengths,self.last_spectra),label="last spectra")).opts(
width=1000, height=600,xlabel="wavelength (nm)",ylabel="reflectance",ylim=(0,1.1))
return (hv.Overlay(curve_list)).opts(width=1000, height=600,xlabel="wavelength (nm)",ylabel="reflectance",ylim=(0,1.1))
@patch
def update_lib_folder(self:SpectralMatcher,directory,sort=False,save=False,show=True):
fnames = sorted(os.listdir(directory))
cwd = Path(directory)
temp = None
for f in fnames:
if "ASD" not in f: continue
if temp is None:
temp = pd.read_csv(cwd/f,delimiter="\t")
temp.rename({temp.columns[0]:f[9:-9]}, axis='columns',inplace=True)
temp.loc[~(temp[temp.columns[0]] > 0), temp.columns[0]]=np.nan
if (np.sum(temp<0.)>0).to_numpy()[0]:
breakpoint()
#col_name = temp.columns[0]
#temp.assign(col_name = lambda x: x.col_name.where(x.col_name.ge(0)))
else:
buff = pd.read_csv(cwd/f,delimiter="\t")
buff.loc[~(buff[buff.columns[0]] > 0), buff.columns[0]]=np.nan
temp.insert(len(temp.columns),f[9:-9],buff[buff.columns[0]])
self.orig_speclib = pd.concat([self.orig_speclib,temp[temp.columns[1:]]],axis=1)
self.orig_speclib = self.orig_speclib.loc[:,~self.orig_speclib.columns.duplicated()]
print(f"Added folder of ASD spectra to spectral library")
self._interp()
return self._sort_save_show(sort,save,show)
spec_matcher = SpectralMatcher(speclib_path="assets/speclib.pkl",model_6SV=model)
spec_matcher.update_lib_folder("../../Downloads/ASCIIdata_splib07b/ChapterV_Vegetation/")
spec_matcher.show(is_ref=True).opts(tools=["hover"])
spec_matcher.orig_speclib["Cactus_Opuntia-1_purple_pad_ASDFRa"].plot()
@patch
def _interp(self:SpectralMatcher):
# interpolate the spectral library to the OpenHSI wavelengths
self.speclib = self.orig_speclib.copy()
self.speclib.insert(0,"type","USGS")
self.speclib = pd.concat( [ pd.DataFrame({"type":"openhsi","wavelength":self.wavelengths}), self.speclib] )
self.speclib.set_index("wavelength",inplace=True)
self.speclib.interpolate(method="cubicspline",axis="index",limit_direction="both",inplace=True)
self.speclib = self.speclib[self.speclib["type"].str.match("openhsi")]
self.speclib.drop("type", 1,inplace=True)
self.speclib_ref = self.speclib.copy()
self.spectra = (self.speclib[:].T*self.model_6SV.radiance/10).T
self.spectra_norm = norm(self.spectra,axis=0)
speclib = spec_matcher.orig_speclib[["wavelength","Cactus_Opuntia-1_purple_pad_ASDFRa"]]
speclib.insert(0,"type","USGS") # insert column in 0th index with label "type" and value "USGS"
speclib = pd.concat( [ pd.DataFrame({"type":"openhsi","wavelength":spec_matcher.wavelengths}), speclib] )
speclib.set_index("wavelength",inplace=True)
speclib.tail(2000)